package com.wzgl.core.webmagic.processor;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import com.jfinal.kit.StrKit;
import com.jfinal.plugin.redis.Cache;
import com.jfinal.plugin.redis.Redis;
import com.wzgl.core.CacheConsts;
import com.wzgl.core.model.TfinishUrl;
import com.wzgl.core.model.Tgrid;
import com.wzgl.core.server.FinishurlService;
import com.wzgl.core.utils.DateUtil;
import com.wzgl.core.utils.WZGLUtils;
import com.wzgl.core.vo.GameSkyVo;
import com.wzgl.core.vo.MeiWenVo;
import com.wzgl.core.webmagic.MagicUtils;
import com.wzgl.core.webmagic.pipeline.TgridsPipeLine;
import com.wzgl.core.webmagic.utils.HrefUtils;
import com.wzgl.core.webmagic.utils.PicUtiles;

/**
 * 游民星空-咨询   每天33篇
 * @author code4crafter@gmail.com <br>
 * @since 0.6.0
 */
public class GamerskyPageProcessor extends GameSkyVo implements PageProcessor,Serializable,Job {
	private static final long serialVersionUID = 1L;
	private static FinishurlService me = FinishurlService.me;
    private static Tgrid t = null;
    private static Spider s ;
    private int size = 1;
    private StringBuffer sb = new StringBuffer();
    private Site site = Site.me().setRetryTimes(1).setSleepTime(1000).addHeader("Accept-Encoding", "/");
    private String pic_ = "http://www.gamersky.com";
    private GameSkyVo tf ;
    
    
    
	@Override
	public void execute(JobExecutionContext context) throws JobExecutionException {
		s = Spider.create(new GamerskyPageProcessor()).addUrl("http://www.gamersky.com/pcgame").thread(1);    
    	s.addPipeline(new TgridsPipeLine());
//    	s.getStatus();
    	s.start();
    	System.out.println("游民星空咨询--------------------------执行完毕\t"+"时间：" +DateUtil.getTime());
	}
    
    @Override
    public void process(Page page) {
    	 if(page.getUrl().regex("http://www.gamersky.com/news/").match() 
    			 || page.getUrl().regex("http://www.gamersky.com/hardware/").match()
    			 || page.getUrl().regex("http://www.gamersky.com/review/").match() 
    			 || page.getUrl().regex("http://www.gamersky.com/handbook/").match()
    			 || page.getUrl().regex("http://www.gamersky.com/zl/column/").match()
    			 || page.getUrl().regex("http://www.gamersky.com/tech/").match()){
    		 String  cont = "";
    		 //在列表结果集中拿出信息
    		 Tgrid tgrid = (Tgrid) WZGLUtils.getCache(CacheConsts.CACHE_MAGIC_GIRD_NAME, page.getUrl());
    		 
    		 //来源
    		 String str = page.getHtml().xpath("//div[@class='detail']").toString();
    		 String ffrom ="";
    		 if(str == null || "".equals(str)){
    			 ffrom = "网络";
    		 }else{
    			 String from = Jsoup.parse(page.getHtml().xpath("//div[@class='detail']").toString())
        				 .select("div").text();
        		 ffrom = from.split(" ")[2].split("：")[1]+"  "+from.split(" ")[3];
        		 
    		 }
    		 System.out.println("来源:  "+ffrom+"\n"); 
    		 tgrid.setFfrom(ffrom);

    		 //正文
    		 String pic_text = page.getHtml().xpath("//div[@class='Mid2L_con']").toString();
    		 if("".equals(pic_text) || pic_text == null){
    			 pic_text = page.getHtml().xpath("//div[@class='MidLcon']").toString();
    		 }
    		 if("".equals(pic_text) || pic_text == null){
    			 cont = "";
    		 }else{
    			 cont = pic_text.substring(pic_text.indexOf("<p>"), pic_text.lastIndexOf("</p>")+4);
    		 }	
    		 
    		 //控制获取分页 
 	       	 List<String> hrefs = page.getHtml().xpath("//div[@class='page_css']/a").all();	 
 	       	 if(hrefs.size()>0){
 	       		String repex = "//div[@class='Mid2L_con']";
 	       		cont = HrefUtils.getOtherHrefs(hrefs, cont,repex);
 	       	 }
    		 
    		 //下载封面图片
    		 List<String> pic_urls = page.getHtml().xpath("//div[@class='Mid2L_con']/p/a/img").all();
    		 List<String> pic_hrefs = page.getHtml().xpath("//div[@class='Mid2L_con']/p/a").all();
    		 if(pic_urls.size()<=0){
    			 pic_urls = page.getHtml().xpath("//div[@class='MidLcon']/p/a/img").all();
    		 }
    		 if(pic_urls.size()<=0){
    			 tgrid.setFcoverImg(null);
    			 tgrid.setFcoverImg2(null);
    		 }else{
    			 tgrid = PicUtiles.downPics(pic_urls, tgrid);
    			 //替换正文中所有的图片和链接
        		 cont = PicUtiles.replaceAllHrefAndImg(pic_hrefs,pic_urls,cont);
    		 }
    		 
    		 System.out.println("cont:  "+cont+"\n");
    		 tgrid.setFhtml(cont);
    		 
    		 //通过url缓存此对象
    		 String cachename = CacheConsts.CACHE_MAGIC_GIRD_NAME;
			 String name = cachename+":"+page.getUrl();
			 WZGLUtils.setCache(cachename, name, tgrid);
			 //更新爬取日志
    		 sb.append("读取第"+size+"详情.................").append("\n\r");
    		 String log = (String) WZGLUtils.getCache(CacheConsts.CACHE_LOG_MAGIC_NAME,"爬取日志:"+pic_);
    		 log += sb.toString();
    		 String log_cache = CacheConsts.CACHE_LOG_MAGIC_NAME;
    		 String log_name = CacheConsts.CACHE_LOG_MAGIC_NAME+":"+pic_;
    		 WZGLUtils.setCache(log_cache, log_name, log);
    		 
    		 size++;
	   	 }else{
	   		 //第一页
	   		 List<String> page_1= page.getHtml().xpath("//div[@class='Mid2_L']/ul[@class='pictxt block']/li").all();
	   		 int num1=1;
	       	 for(int i = 0;i<page_1.size();i++){
	       		 t = new Tgrid();
	       		 String li = page_1.get(i);
	       		 Document docList = Jsoup.parse(li);
	       		 //网页详情
	       		 String href = docList.getElementsByClass("img").select("a").attr("href");
	       		 //列表中显示的图片地址
	       		 String img = docList.getElementsByClass("img").select("img").attr("src");
	       		 //列表标题
	       		 String title = docList.getElementsByClass("tit").select("a").attr("title");
	       		 //列表摘要
	       		 String desc = docList.getElementsByClass("txt").text();
	       		 //列表发布时间
	       		 String time = docList.getElementsByClass("time").text();

	       		 //是否已经爬取过
	       		 String cn = CacheConsts.CACHE_MAGIC_GIRD_NAME; 
	       		 String key = CacheConsts.CACHE_MAGIC_GIRD_NAME+":"+href;
	       		 if(!WZGLUtils.exists(cn, key)){
	       			System.out.println("地址："+href);
			       	System.out.println("标题："+title);
			       	System.out.println("封面："+img);
			        System.out.println("时间："+time);
			       	System.out.println("摘要："+desc+"\n");
	       			tf = new GameSkyVo();
	       			tf.setFdescribe("游民星空-咨询");
	       			tf.setFmzgicId("afadfafaf1123123");
	       			tf.setFtype("100008");
	       			//保存到对象
			       	t.setFtitle(title);t.setFdate(new Date());t.setFtitle(title);
			       	t.setFaddress(href);t.setFgetDate(time);t.setFdescribe(tf.getFdescribe());
			       	t.setFtype(tf.getFtype());t.setFmzgicId(tf.getFmzgicId());t.setFstatus(0);
			        //把爬取了的url放入缓存
			       	String cachename = CacheConsts.CACHE_MAGIC_GIRD_NAME;
					String name = cachename+":"+href;
					WZGLUtils.setCache(cachename, name, t);
		       		 //记录爬取日志
	        		 sb.append("第"+num1+"文章开始读取----------------->\r");
	        		 sb.append("第"+num1+"标题:"+title).append("\r");
	        		 sb.append("第"+num1+"文章详情URL:"+href).append("\r");
	        		 sb.append("第"+num1+"文章时间:"+time).append("\r");
	        		 sb.append("第"+num1+"文章开始结束----------------->").append("\r");
	        		 //加入缓存
	        		 String log = Redis.use(CacheConsts.CACHE_LOG_MAGIC_NAME).get(pic_);
	        		 log += sb.toString();
	        		 String log_cache = CacheConsts.CACHE_LOG_MAGIC_NAME;
	        		 String log_name = CacheConsts.CACHE_LOG_MAGIC_NAME+":"+pic_;
	        		 WZGLUtils.setCache(log_cache, log_name, log);
	        		 //加入数据库
	        		 TfinishUrl tf = new TfinishUrl();
	        		 tf.setFurl(href);
	        		 tf.setFdate(new Date());
	        		 me.save(tf);
	        		 
		       		 page.setSkip(true);
		       		 page.addTargetRequest(href);
	       		 }
	       		 num1++;
	       	 }
	       	 
	       	 //第二页,第三页
	       	 List<String> page_2= page.getHtml().xpath("//div[@class='Mid2_L']/ul[@class='pictxt none']/li").all();
	       	 for(String li:page_2){
	       		t = new Tgrid();  		 
	       		 Document docList = Jsoup.parse(li);
	       		 String href = docList.getElementsByClass("img").select("a").attr("href");
	       		 //列表中显示的图片地址
	       		 String img = docList.getElementsByClass("img").select("img").attr("src");
	       		 //列表标题
	       		 String title = docList.getElementsByClass("tit").select("a").attr("title");
	       		 //列表摘要
	       		 String desc = docList.getElementsByClass("txt").text();
	       		 //列表发布时间
	       		 String time = docList.getElementsByClass("time").text();
	       		 //是否已经爬取过
	       		 String cn = CacheConsts.CACHE_MAGIC_GIRD_NAME; 
	       		 String key = CacheConsts.CACHE_MAGIC_GIRD_NAME+":"+href;
	       		 if(!WZGLUtils.exists(cn, key)){
	       			 //保存到对象
	       			System.out.println("地址："+href);
			       	System.out.println("标题："+title);
			       	System.out.println("封面："+img);
			        System.out.println("时间："+time);
			       	System.out.println("摘要："+desc+"\n");
	       			t.setFtitle(title);t.setFdate(new Date());t.setFtitle(title);
			       	t.setFaddress(href);t.setFgetDate(time);t.setFdescribe(tf.getFdescribe());
			       	t.setFtype(tf.getFtype());t.setFmzgicId(tf.getFmzgicId());t.setFstatus(0);
		       		 String cachename = CacheConsts.CACHE_MAGIC_GIRD_NAME;
					 String name = cachename+":"+href;
					 WZGLUtils.setCache(cachename, name, t);
		       		 //记录爬取日志
	        		 sb.append("第"+num1+"文章开始读取----------------->\r");
	        		 sb.append("第"+num1+"标题:"+title).append("\r");
	        		 sb.append("第"+num1+"文章详情URL:"+href).append("\r");
	        		 sb.append("第"+num1+"文章时间:"+time).append("\r");
	        		 sb.append("第"+num1+"文章开始结束----------------->").append("\r");
	        		 //加入缓存
	        		 String log = Redis.use(CacheConsts.CACHE_LOG_MAGIC_NAME).get(pic_);
	        		 log += sb.toString();
	        		 String log_cache = CacheConsts.CACHE_LOG_MAGIC_NAME;
	        		 String log_name = CacheConsts.CACHE_LOG_MAGIC_NAME+":"+pic_;
	        		 WZGLUtils.setCache(log_cache, log_name, log);
	        		 //加入数据库
	        		 TfinishUrl tf = new TfinishUrl();
	        		 tf.setFurl(href);
	        		 tf.setFdate(new Date());
	        		 me.save(tf);

		       		 page.setSkip(true);
		       		 page.addTargetRequest(href);
	       		 }
	       		 num1++;
	       	 }
	   	 }
    }

    @Override
    public Site getSite() {
        return site;
    }

}